In [1]:
# Creating the entry point for spark
from pyspark import SparkContext,SQLContext
sc = SparkContext()
sqlContext = SQLContext(sc)
from pyspark.sql import SparkSession
sparkSession = SparkSession.builder.getOrCreate()
# Importing the below modules for data manipulation
from pyspark.sql.functions import count
In [2]:
# To handle matrix & array computations
import numpy as np
# to handle data in form of rows and columns
import pandas as pd
# importing ploting libraries
import matplotlib.pyplot as plt
from scipy.stats import itemfreq
# For visualization
import seaborn as sns
# For train & test split
from sklearn.model_selection import train_test_split
# For plotting inline in Jupyter notebook
%matplotlib inline
# datetime processing
from datetime import datetime
# Data Preprocessing
from sklearn import preprocessing
# For conducting statistical tests
import statsmodels.api as sm
import statsmodels.formula.api as smf
# For creating regression Models using LinearRegression, Stochastic Gradient Descent regression, Decision Tree, RandomForest
from sklearn.linear_model import LinearRegression
from sklearn.linear_model import SGDRegressor
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
# For calculating cross validation score
from sklearn.model_selection import cross_val_score
# For calculating different metrics like R2, mean abolute error and mean squared error
from sklearn.metrics import make_scorer, mean_absolute_error, mean_squared_error
# For identifying and setting best hypertuning parameters for the model
from sklearn.model_selection import GridSearchCV
# For creating decision tree in graphical mode
import graphviz
from sklearn.tree import export_graphviz
# special matplotlib argument for improved plots
from matplotlib import rcParams
sns.set_style("whitegrid")
sns.set_context("poster")
In [3]:
# Hypothesis
# Oil price plays a role in the total number of sales transactions & sales
In [4]:
# Loading Merged train daily transactions for store 44
In [5]:
sdfTrain = sqlContext.read.csv("/gl-capstone-data/Team6-C-Sep/Data/train_transactions_daily_store44.csv",header = True,inferSchema = True)
In [6]:
sdfTrain.printSchema()
In [7]:
sdfTrain.count()
Out[7]:
In [8]:
print(sdfTrain.show(5))
In [9]:
sqlContext.registerDataFrameAsTable(sdfTrain, "sdfTrainTbl")
Merge Train and Oil:
In [10]:
sdfOil = sqlContext.read.csv("/gl-capstone-data/Team6-C-Sep/Data/oil_2013_2015_interpolated.csv",header = True,inferSchema = True)
In [11]:
sdfOil.printSchema()
In [12]:
sdfOil.count()
Out[12]:
In [13]:
minRow = sdfOil.agg({"date": "min"}).collect()[0]
print minRow["min(date)"]
In [14]:
maxRow = sdfOil.agg({"date": "max"}).collect()[0]
print maxRow["max(date)"]
In [15]:
sqlContext.registerDataFrameAsTable(sdfOil, "sdfOilTbl")
In [16]:
# Merge Training dataset with interpolated oil dataset
In [17]:
sdfTrainWithOil = sdfTrain.join(sdfOil, ["date"],"leftouter")
In [18]:
print(sdfTrainWithOil.show(20))
In [19]:
sdfTrainWithOil.count()
Out[19]:
Merge Train and Holiday_events:
In [20]:
sdfHolidayEvents = sqlContext.read.csv("/gl-capstone-data/Team6-C-Sep/Data/holidays_events_2013_2015_noduplicates.csv",header = True,inferSchema = True)
In [21]:
print(sdfHolidayEvents.show(5))
In [22]:
sdfHolidayEvents.count()
Out[22]:
In [23]:
sqlContext.registerDataFrameAsTable(sdfHolidayEvents, "sdfHolidayEvents")
In [24]:
sdfHolidayEvents.printSchema()
In [25]:
sdfTrainWithOilHolidayEventsJoined = sdfTrainWithOil.join(sdfHolidayEvents, ["date"],"leftouter")
In [26]:
sqlContext.registerDataFrameAsTable(sdfTrainWithOilHolidayEventsJoined, "sdfTrainWithOilHolidayEventsJoined")
In [27]:
sqlContext.sql("select count(*) from sdfTrainWithOilHolidayEventsJoined").show()
In [28]:
minRow_Train = sdfTrainWithOilHolidayEventsJoined.agg({"date": "min"}).collect()[0]
print minRow_Train["min(date)"]
In [29]:
maxRow_Train = sdfTrainWithOilHolidayEventsJoined.agg({"date": "max"}).collect()[0]
print maxRow_Train["max(date)"]
In [30]:
#date_mask = (sdfTrainWithOilHolidayEventsJoined['date'] >= '2015-01-01') & (sdfTrainWithOilHolidayEventsJoined['date'] <= '2015-12-31')
pd_train = sdfTrainWithOilHolidayEventsJoined.toPandas()
#Print the size
len(pd_train)
Out[30]:
In [31]:
pd_train.head(5)
Out[31]:
Data Pre-processing¶
In [32]:
pd_train_nan = (pd_train.isnull().sum() / pd_train.shape[0]) * 100
pd_train_nan
Out[32]:
In [33]:
# There are 86% of Nulls/NA in Holiday events attributes. Replacing Nulls or NA with No_Holiday as the default value
In [34]:
pd_train['type'] = pd_train.type.replace(np.NaN, 'No_Holiday')
pd_train['locale'] = pd_train.locale.replace(np.NaN, 'None')
pd_train['locale_name'] = pd_train.locale_name.replace(np.NaN, 'None')
pd_train['description'] = pd_train.description.replace(np.NaN, 'None')
pd_train['transferred'] = pd_train.transferred.replace(np.NaN, 'None')
In [35]:
# Rechecking for Nulls, No Nulls now
In [36]:
pd_train_nan = (pd_train.isnull().sum() / pd_train.shape[0]) * 100
pd_train_nan
Out[36]:
Generating Summary Statistics¶
In [37]:
# Inference
# 1089 Observations, 10 features
# Descriptive statistics reveal: average transaction volume is 4133 between 2013 - 2015 & Monthly average sales is $30586 for store 44 for the period 2013 - 2015
# Minimum transaction volume is 2333 & Maximum transaction volume is 8359.Min sales is $9067 and Max sales is $78070
# Oil price range from $34 to $110
In [38]:
#Shape
print('Shape : ', pd_train.shape, '\n')
#Type
print('Type : ', '\n', pd_train.dtypes)
#Summary
pd_train.describe()
Out[38]:
In [39]:
pd_train.sample(10)
Out[39]:
In [40]:
pd_train.type.unique()
Out[40]:
In [41]:
pd_train.description.unique()
Out[41]:
In [42]:
pd_train.locale.unique()
Out[42]:
In [43]:
pd_train.transferred.unique()
Out[43]:
Data Pre-Processing¶
In [44]:
# Formatting date to YYYY-MM-DD
In [45]:
pd_train['date']=pd_train['date'].apply(lambda x: x.strftime('%Y-%m-%d'))
In [46]:
pd_train['date'].sample(10)
Out[46]:
In [47]:
pd_train.sample(10)
Out[47]:
In [48]:
# Reformat the date - Get Month Year
def get_month_year(df):
df['month'] = df.date.apply(lambda x: x.split('-')[1])
df['year'] = df.date.apply(lambda x: x.split('-')[0])
return df
get_month_year(pd_train);
In [49]:
pd_train['date'] = pd.to_datetime(pd_train['date'])
pd_train['day'] = pd_train['date'].dt.weekday_name
pd_train = pd_train.drop('date', axis=1)
In [50]:
pd_train.sample(10)
Out[50]:
In [51]:
dummy_variables = ['type','store_nbr','locale', 'locale_name','transferred', 'month', 'day']
for var in dummy_variables:
dummy = pd.get_dummies(pd_train[var], prefix = var, drop_first = False)
pd_train = pd.concat([pd_train, dummy], axis = 1)
pd_train = pd_train.drop(dummy_variables, axis = 1)
pd_train = pd_train.drop(['year'], axis = 1)
In [52]:
pd_train=pd_train.drop("description",axis=1)
In [53]:
pd_train.sample(10)
Out[53]:
In [54]:
sns.regplot(x='dcoilwtico',
y='transactions',
data=pd_train,
scatter_kws={'alpha':0.3},
line_kws={'color':'black'})
Out[54]:
In [55]:
sns.regplot(x='dcoilwtico',
y='unit_sales',
data=pd_train,
scatter_kws={'alpha':0.3},
line_kws={'color':'black'})
Out[55]:
In [56]:
#Re-scale Sales, Transactions, Oil Price using the standard scaler
scaler = preprocessing.StandardScaler()
pd_train['unit_sales'] = scaler.fit_transform(pd_train['unit_sales'].reshape(-1,1))
pd_train['dcoilwtico'] = scaler.fit_transform(pd_train['dcoilwtico'].reshape(-1,1))
pd_train['transactions'] = scaler.fit_transform(pd_train['transactions'].reshape(-1,1))
In [57]:
pd_train.sample(10)
Out[57]:
In [58]:
print('Shape : ', pd_train.shape)
Exploratory Data Analysis for store 44 with the prepared dataset¶
In [59]:
# The features highly correlated with transactions is Unit sales, Remove unit sales from the dependent variables.
# As the number of sales increases, total sales also increases and vice versa. Hence they are highly correlated and sales can be removed.
In [60]:
import matplotlib.pyplot as plt
corr = pd_train.corr()
corr.style.background_gradient()
Out[60]:
In [61]:
plt.figure(figsize=(32,32))
plt.matshow(pd_train.corr(), cmap=plt.cm.Reds, fignum=1)
plt.colorbar()
tick_marks = [i for i in range(len(pd_train.columns))]
plt.xticks(tick_marks, pd_train.columns, rotation=90)
plt.yticks(tick_marks, pd_train.columns)
Out[61]:
In [62]:
# Keeping only the highly correlated features for plotting the distribution
pd_train_filtered = pd_train[['transactions','dcoilwtico','unit_sales','type_Additional','month_12','day_Saturday','day_Sunday']]
In [63]:
# Check the data distribution through pairplot
sns.pairplot(pd_train_filtered)
Out[63]:
In [64]:
sns.regplot(x='dcoilwtico',
y='transactions',
data=pd_train,
scatter_kws={'alpha':0.3},
line_kws={'color':'black'})
Out[64]:
In [65]:
sns.regplot(x='dcoilwtico',
y='unit_sales',
data=pd_train,
scatter_kws={'alpha':0.3},
line_kws={'color':'black'})
Out[65]:
In [66]:
sns.regplot(x='transactions',
y='unit_sales',
data=pd_train,
scatter_kws={'alpha':0.3},
line_kws={'color':'black'})
Out[66]:
Checking for statistical significance¶
Is there any statistically significant relation between Oil price and Transaction Volume for the store 44 ?
Null Hypothesis H0 = Oil price and Transaction Volume are independent from each other.
Alternative Hypothesis HA = Oil price and Transaction Volume are not independent of each other. There is a relationship between them.
Oil Price - Independent continuous variable
Transaction Volume - Dependent continuous variable
In [67]:
lin_model = smf.ols(formula = 'transactions ~ dcoilwtico', data = pd_train).fit()
In [68]:
#print the summary
print(lin_model.summary())
In [69]:
# Inference
# p-value is < 0.05 and hence there is relationship between transaction volume and oil price.
# So, rejecting the null hypothesis
Is there any statistically significant relation between Oil price and Sales for the store 44 ?
Null Hypothesis H0 = Oil price and sales are independent from each other.
Alternative Hypothesis HA = Oil price and sales are not independent of each other. There is a relationship between them.
Oil Price - Independent continuous variable
Sales - Dependent continuous variable
In [70]:
lin_model = smf.ols(formula = 'unit_sales ~ dcoilwtico', data = pd_train).fit()
In [71]:
#print the summary
print(lin_model.summary())
In [72]:
# Inference
# p-value is < 0.05 and hence there is a relationship between oil price and sales.
# So, rejecting the null hypothesis
Train Test split¶
In [73]:
# Modeling for Transactions as the dependent variable and all the other variables from the filtered dataframe except total sales as the independent variables.
In [74]:
X_train = pd_train.drop(['unit_sales','transactions'], axis = 1)
y_labels = pd_train['transactions']
In [75]:
num_test = 0.35
X_train, X_test, y_train, Y_test = train_test_split(X_train, y_labels, test_size=num_test, random_state=15)
print('X_train shape :', X_train.shape)
print('y_train shape :', y_train.shape)
print('X_test shape :', X_test.shape)
print('y_test shape :', Y_test.shape)
Transaction Volume Prediction¶
In [76]:
regression_model = LinearRegression()
regression_model.fit(X_train.as_matrix(), y_train)
Out[76]:
In [77]:
print(regression_model.coef_)
print(regression_model.intercept_)
In [78]:
# Checking the coefficients for each of the independent attributes
for idx, col_name in enumerate(X_train.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[idx]))
In [79]:
# Let us check the intercept for the model
intercept = regression_model.intercept_
print("The intercept for the model is {}".format(intercept))
In [80]:
regression_model.score(X_test, Y_test)
Out[80]:
In [81]:
# checking the sum of squared errors by predicting value of y for test cases and
# subtracting from the actual y for the test cases
mse = np.mean((regression_model.predict(X_test)-Y_test)**2)
In [82]:
# underroot of mean_sq_error is standard deviation i.e. avg variance between predicted and actual
import math
math.sqrt(mse)
Out[82]:
In [83]:
# predict transaction volume for a set of attributes
y_pred = regression_model.predict(X_test)
In [84]:
# Since this is regression, plot the predicted y value vs actual y values for the test data
plt.scatter(Y_test, y_pred)
Out[84]:
In [85]:
from sklearn.metrics import r2_score
R2_Lin_Reg_Tran_Pred = r2_score(Y_test, y_pred)
print(R2_Lin_Reg_Tran_Pred)
In [86]:
y_train_pred = regression_model.predict(X_train)
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Train data
print(mean_absolute_error(y_train, y_train_pred))
print(mean_squared_error(y_train, y_train_pred))
print(np.sqrt(mean_squared_error(y_train, y_train_pred)))
In [87]:
RMSE_Lin_Reg_Tran_Pred = np.sqrt(mean_squared_error(Y_test, y_pred))
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of simple linear regression model is : " + str(mean_absolute_error(Y_test, y_pred)))
print("The mean squared error of simple linear regression model is : " + str(mean_squared_error(Y_test, y_pred)))
print("The root mean squared error of simple linear regression model is : " + str(RMSE_Lin_Reg_Tran_Pred))
Model 1b: Linear regression using stochastic gradient descent without regularization¶
In [88]:
sgd_reg = SGDRegressor(n_iter=50, penalty=None, eta0=0.1)
sgd_reg.fit(X_train, y_train.ravel())
Out[88]:
In [89]:
print(sgd_reg.intercept_)
print(sgd_reg.coef_)
In [90]:
# Checking the coefficients for each of the independent attributes
for idx, col_name in enumerate(X_train.columns):
print("The coefficient for {} is {}".format(col_name, sgd_reg.coef_[idx]))
In [91]:
sgd_reg.score(X_test, Y_test)
Out[91]:
In [92]:
# checking the sum of squared errors by predicting value of y for test cases and
# subtracting from the actual y for the test cases
mse = np.mean((sgd_reg.predict(X_test)-Y_test)**2)
In [93]:
# underroot of mean_sq_error is standard deviation i.e. avg variance between predicted and actual
import math
math.sqrt(mse)
Out[93]:
In [94]:
# predict transaction volume for a set of attributes
y_pred = sgd_reg.predict(X_test)
In [95]:
# Since this is regression, plot the predicted y value vs actual y values for the test data
plt.scatter(Y_test, y_pred)
Out[95]:
In [96]:
from sklearn.metrics import r2_score
R2_Sgd_Tran_Pred = r2_score(Y_test, y_pred)
print(R2_Sgd_Tran_Pred)
In [97]:
y_train_pred = sgd_reg.predict(X_train)
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Train data
print(mean_absolute_error(y_train, y_train_pred))
print(mean_squared_error(y_train, y_train_pred))
print(np.sqrt(mean_squared_error(y_train, y_train_pred)))
In [98]:
RMSE_Sgd_Tran_Pred = np.sqrt(mean_squared_error(Y_test, y_pred))
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of simple linear regression model is : " + str(mean_absolute_error(Y_test, y_pred)))
print("The mean squared error of simple linear regression model is : " + str(mean_squared_error(Y_test, y_pred)))
print("The root mean squared error of simple linear regression model is : " + str(RMSE_Sgd_Tran_Pred))
Model 2 : Decision Tree with and without gridsearch CV¶
In [99]:
# Criterion : Mean absolute error
In [100]:
dtree = DecisionTreeRegressor(random_state=0, criterion="mae")
dtree_fit = dtree.fit(X_train, y_train)
In [101]:
dtree_scores = cross_val_score(dtree_fit, X_train, y_train, cv = 5)
print("mean cross validation score: {}".format(np.mean(dtree_scores)))
print("score without cv: {}".format(dtree_fit.score(X_train, y_train)))
In [102]:
# on the test or hold-out set
from sklearn.metrics import r2_score
y_pred = dtree_fit.predict(X_test)
R2_DT_WoutGCV_Tran_Pred = r2_score(Y_test, y_pred)
print(R2_DT_WoutGCV_Tran_Pred)
print(dtree_fit.score(X_test, Y_test))
In [103]:
final_mae = mean_absolute_error(Y_test, y_pred)
final_mse = mean_squared_error(Y_test, y_pred)
final_rmse = np.sqrt(final_mse)
In [104]:
RMSE_DT_WoutGCV_Tran_Pred = final_rmse
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of decision tree without gridsearchcv is : " + str(final_mae))
print("The mean squared error of decision tree without gridsearchcv is : " + str(final_mse))
print("The root mean squared error of decision tree without gridsearchcv is : " + str(final_rmse))
In [105]:
scoring = make_scorer(r2_score)
g_cv = GridSearchCV(DecisionTreeRegressor(random_state=0),
param_grid={'min_samples_split': range(2, 10)},
scoring=scoring, cv=5, refit=True)
g_cv.fit(X_train, y_train)
result = g_cv.cv_results_
# print(result)
R2_DT_WithGCV_Tran_Pred = r2_score(Y_test, g_cv.best_estimator_.predict(X_test))
print(R2_DT_WithGCV_Tran_Pred)
In [106]:
y_pred = g_cv.best_estimator_.predict(X_test)
In [107]:
final_mae = mean_absolute_error(Y_test, y_pred)
final_mse = mean_squared_error(Y_test, y_pred)
final_rmse = np.sqrt(final_mse)
In [108]:
RMSE_DT_WithGCV_Tran_Pred = final_rmse
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of decision tree with gridsearchcv is : " + str(final_mae))
print("The mean squared error of decision tree with gridsearchcv is : " + str(final_mse))
print("The root mean squared error of decision tree with gridsearchcv is : " + str(final_rmse))
In [109]:
print("The best params from gridsearchcv are :" + str(g_cv.best_params_))
In [110]:
print("The best estimators from gridsearchcv are :" + str(g_cv.best_estimator_))
In [111]:
# Printing the decision tree
dot_data = export_graphviz(dtree_fit, out_file=None)
In [112]:
graph = graphviz.Source(dot_data)
In [113]:
dot_data = export_graphviz(dtree_fit, out_file=None,
feature_names=X_train.columns,
filled=True, rounded=True,
special_characters=True)
In [114]:
graph = graphviz.Source(dot_data)
graph
# Inference:
# Oil price is the main predictor value using which decision tree is built and is an influencing factor in the total transaction volume.
Out[114]:
Model 3 : Random Forest with GridsearchCV¶
In [115]:
# Choose the type of classifier.
RFR = RandomForestRegressor()
# Choose some parameter combinations to try
parameters = {'n_estimators': [5, 10, 100],
'criterion': ['mse','mae'],
'max_depth': [5, 10, 15],
'min_samples_split': [2, 5, 10],
'min_samples_leaf': [1,5]
}
# Type of scoring used to compare parameter combinations
# Run the grid search
grid_obj = GridSearchCV(RFR, parameters,
cv=5, #Determines the cross-validation splitting strategy /to specify the number of folds in a (Stratified)KFold
n_jobs=-1, #Number of jobs to run in parallel
verbose=1)
grid_obj = grid_obj.fit(X_train, y_train)
# Set the clf to the best combination of parameters
RFR = grid_obj.best_estimator_
# Fit the best algorithm to the data.
RFR.fit(X_train, y_train)
Out[115]:
In [116]:
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
predictions = RFR.predict(X_test)
R2_RF_WithGCV_Tran_Pred = r2_score(Y_test, predictions)
print('R2 score = ',R2_RF_WithGCV_Tran_Pred, '/ 1.0')
print("The mean absolute error of random forest with gridsearchcv is : " + str(mean_absolute_error(Y_test, predictions)))
print("The mean squared error of random forest with gridsearchcv is : " + str(mean_squared_error(Y_test, predictions)))
print("The root mean squared error of random forest with gridsearchcv is : " + str(np.sqrt(mean_squared_error(Y_test, predictions))))
In [117]:
RMSE_RF_WithGCV_Tran_Pred = np.sqrt(mean_squared_error(Y_test, predictions))
In [118]:
#Check and plot the 500 first predictions
plt.plot(Y_test.as_matrix()[0:500], '+', color ='red', alpha=0.7)
plt.plot(predictions[0:500], 'ro', color ='green', alpha=0.5)
plt.show()
Sales Prediction¶
In [119]:
# Modeling for Total sales as the dependent variable and all the other variables except transactions as the independent variables.
In [120]:
X_train = pd_train.drop(['unit_sales','transactions'], axis = 1)
y_labels = pd_train['unit_sales']
In [121]:
num_test = 0.20
X_train, X_test, y_train, Y_test = train_test_split(X_train, y_labels, test_size=num_test, random_state=15)
print('X_train shape :', X_train.shape)
print('y_train shape :', y_train.shape)
print('X_test shape :', X_test.shape)
print('Y_test shape :', Y_test.shape)
In [122]:
regression_model = LinearRegression()
regression_model.fit(X_train.as_matrix(), y_train)
Out[122]:
In [123]:
print(regression_model.coef_)
print(regression_model.intercept_)
In [124]:
# Checking the coefficients for each of the independent attributes
for idx, col_name in enumerate(X_train.columns):
print("The coefficient for {} is {}".format(col_name, regression_model.coef_[idx]))
In [125]:
# Let us check the intercept for the model
intercept = regression_model.intercept_
print("The intercept for the model is {}".format(intercept))
In [126]:
regression_model.score(X_test, Y_test)
Out[126]:
In [127]:
# checking the sum of squared errors by predicting value of y for test cases and
# subtracting from the actual y for the test cases
mse = np.mean((regression_model.predict(X_test)-Y_test)**2)
In [128]:
# underroot of mean_sq_error is standard deviation i.e. avg variance between predicted and actual
import math
math.sqrt(mse)
Out[128]:
In [129]:
# predict transaction volume for a set of attributes
y_pred = regression_model.predict(X_test)
In [130]:
# Since this is regression, plot the predicted y value vs actual y values for the test data
plt.scatter(Y_test, y_pred)
Out[130]:
In [131]:
from sklearn.metrics import r2_score
R2_Lin_Reg_Sale_Pred = r2_score(Y_test, y_pred)
print(R2_Lin_Reg_Sale_Pred)
In [132]:
y_train_pred = regression_model.predict(X_train)
RMSE_Lin_Reg_Sale_Pred = np.sqrt(mean_squared_error(y_train, y_train_pred))
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Train data
print(mean_absolute_error(y_train, y_train_pred))
print(mean_squared_error(y_train, y_train_pred))
print(RMSE_Lin_Reg_Sale_Pred)
In [133]:
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of simple linear regression model is : " + str(mean_absolute_error(Y_test, y_pred)))
print("The mean squared error of simple linear regression model is : " + str(mean_squared_error(Y_test, y_pred)))
print("The root mean squared error of simple linear regression model is : " + str(np.sqrt(mean_squared_error(Y_test, y_pred))))
Model 1b: Linear regression using stochastic gradient descent without regularization¶
In [134]:
sgd_reg = SGDRegressor(n_iter=50, penalty=None, eta0=0.1)
sgd_reg.fit(X_train, y_train.ravel())
Out[134]:
In [135]:
print(sgd_reg.intercept_)
print(sgd_reg.coef_)
In [136]:
sgd_reg.score(X_test, Y_test)
Out[136]:
In [137]:
# checking the sum of squared errors by predicting value of y for test cases and
# subtracting from the actual y for the test cases
mse = np.mean((sgd_reg.predict(X_test)-Y_test)**2)
In [138]:
# underroot of mean_sq_error is standard deviation i.e. avg variance between predicted and actual
import math
math.sqrt(mse)
Out[138]:
In [139]:
# predict transaction volume for a set of attributes
y_pred = sgd_reg.predict(X_test)
In [140]:
# Since this is regression, plot the predicted y value vs actual y values for the test data
plt.scatter(Y_test, y_pred)
Out[140]:
In [141]:
from sklearn.metrics import r2_score
R2_Sgd_Sale_Pred = r2_score(Y_test, y_pred)
print(R2_Sgd_Sale_Pred)
In [142]:
y_train_pred = sgd_reg.predict(X_train)
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Train data
print(mean_absolute_error(y_train, y_train_pred))
print(mean_squared_error(y_train, y_train_pred))
print(np.sqrt(mean_squared_error(y_train, y_train_pred)))
In [143]:
RMSE_Sgd_Sale_Pred = np.sqrt(mean_squared_error(Y_test, y_pred))
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of simple linear regression model with SGD is : " + str(mean_absolute_error(Y_test, y_pred)))
print("The mean squared error of simple linear regression model with SGD is : " + str(mean_squared_error(Y_test, y_pred)))
print("The root mean squared error of simple linear regression model with SGD is : " + str(RMSE_Sgd_Sale_Pred))
Model 2 : Decision Tree with and without gridsearch CV¶
In [144]:
# Criterion : Mean absolute error
In [145]:
dtree = DecisionTreeRegressor(random_state=0, criterion="mae")
dtree_fit = dtree.fit(X_train, y_train)
In [146]:
dtree_scores = cross_val_score(dtree_fit, X_train, y_train, cv = 5)
print("mean cross validation score: {}".format(np.mean(dtree_scores)))
print("score without cv: {}".format(dtree_fit.score(X_train, y_train)))
In [147]:
y_pred = dtree_fit.predict(X_test)
In [148]:
# on the test or hold-out set
from sklearn.metrics import r2_score
R2_DT_WoutGCV_Sale_Pred = r2_score(Y_test,y_pred)
print(R2_DT_WoutGCV_Sale_Pred)
print(dtree_fit.score(X_test, Y_test))
In [149]:
final_mae = mean_absolute_error(Y_test, y_pred)
final_mse = mean_squared_error(Y_test, y_pred)
final_rmse = np.sqrt(final_mse)
In [150]:
RMSE_DT_WoutGCV_Sale_Pred = final_rmse
In [151]:
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of decision tree without gridsearchcv is : " + str(final_mae))
print("The mean squared error of decision tree without gridsearchcv is : " + str(final_mse))
print("The root mean squared error of decision tree without gridsearchcv is : " + str(final_rmse))
In [152]:
scoring = make_scorer(r2_score)
g_cv = GridSearchCV(DecisionTreeRegressor(random_state=0),
param_grid={'min_samples_split': range(2, 10)},
scoring=scoring, cv=5, refit=True)
g_cv.fit(X_train, y_train)
g_cv.best_params_
result = g_cv.cv_results_
# print(result)
R2_DT_WithGCV_Sale_Pred = r2_score(Y_test, g_cv.best_estimator_.predict(X_test))
print(R2_DT_WithGCV_Sale_Pred)
In [153]:
y_pred = g_cv.best_estimator_.predict(X_test)
In [154]:
final_mae = mean_absolute_error(Y_test, y_pred)
final_mse = mean_squared_error(Y_test, y_pred)
final_rmse = np.sqrt(final_mse)
In [155]:
RMSE_DT_WithGCV_Sale_Pred=final_rmse
In [156]:
# Print the results of MAE (mean absolute error), MSE (Mean squared Error), RMSE (Root Mean squared error) - Test data
print("The mean absolute error of decision tree with gridsearchcv is : " + str(final_mae))
print("The mean squared error of decision tree with gridsearchcv is : " + str(final_mse))
print("The root mean squared error of decision tree with gridsearchcv is : " + str(final_rmse))
In [157]:
print("The best params from gridsearchcv are :" + str(g_cv.best_params_))
In [158]:
print("The best estimators from gridsearchcv are :" + str(g_cv.best_estimator_))
In [159]:
# Printing the decision tree
dot_data = export_graphviz(dtree_fit, out_file=None)
In [160]:
graph = graphviz.Source(dot_data)
In [161]:
dot_data = export_graphviz(dtree_fit, out_file=None,
feature_names=X_train.columns,
filled=True, rounded=True,
special_characters=True)
In [162]:
graph = graphviz.Source(dot_data)
graph
Out[162]:
Model 4: Sales Prediction with Random Forest & GridsearchCV¶
In [163]:
from sklearn.ensemble import RandomForestRegressor
from sklearn.model_selection import GridSearchCV
# Choose the type of classifier.
RFR = RandomForestRegressor()
# Choose some parameter combinations to try
#YOU CAN TRY DIFFERENTS PARAMETERS TO FIND THE BEST MODEL
parameters = {'n_estimators': [5, 10, 100],
'criterion': ['mse','mae'],
'max_depth': [5, 10, 15],
'min_samples_split': [2, 5, 10],
'min_samples_leaf': [1,5]
}
# Type of scoring used to compare parameter combinations
# Run the grid search
grid_obj = GridSearchCV(RFR, parameters,
cv=5, #Determines the cross-validation splitting strategy /to specify the number of folds in a (Stratified)KFold
n_jobs=-1, #Number of jobs to run in parallel
verbose=1)
grid_obj = grid_obj.fit(X_train, y_train)
# Set the clf to the best combination of parameters
RFR = grid_obj.best_estimator_
# Fit the best algorithm to the data.
RFR.fit(X_train, y_train)
Out[163]:
In [164]:
print("The best params from gridsearchcv are :" + str(grid_obj.best_params_))
print("The best estimators from gridsearchcv are :" + str(grid_obj.best_estimator_))
In [165]:
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error
predictions = RFR.predict(X_test)
R2_RF_WithGCV_Sale_Pred = r2_score(Y_test, predictions)
print('R2 score = ',R2_RF_WithGCV_Sale_Pred, '/ 1.0')
print('MSE score = ',mean_squared_error(Y_test, predictions), '/ 0.0')
In [166]:
RMSE_RF_WithGCV_Sale_Pred = np.sqrt(mean_squared_error(Y_test, predictions))
In [167]:
print("The mean absolute error of random forest with gridsearchcv is : " + str(mean_absolute_error(Y_test, predictions)))
print("The mean squared error of random forest with gridsearchcv is : " + str(mean_squared_error(Y_test, predictions)))
print("The root mean squared error of random forest with gridsearchcv is : " + str(RMSE_RF_WithGCV_Sale_Pred))
In [168]:
#Check and plot the 500 first predictions
plt.plot(Y_test.as_matrix()[0:500], '+', color ='blue', alpha=0.7)
plt.plot(predictions[0:500], 'ro', color ='red', alpha=0.5)
plt.show()
Results¶
Transaction Prediction: Different Models: Metrics : R-squared </u>
In [169]:
print("Linear regression : R2 score : " + str(R2_Lin_Reg_Tran_Pred))
print("Linear regression with stochastic gradient descent: R2 score : " + str(R2_Sgd_Tran_Pred))
print("Decision Tree without GridsearchCV : R2 score : " + str(R2_DT_WoutGCV_Tran_Pred))
print("Decision Tree with GridsearchCV : R2 score : " + str(R2_DT_WithGCV_Tran_Pred))
print("Random Forest with GridsearchCV : R2 score : "+ str(R2_RF_WithGCV_Tran_Pred))
Sales Prediction: Different Models: Metrics: R-Squared</u>
In [170]:
print("Linear regression : R2 score : " + str(R2_Lin_Reg_Sale_Pred))
print("Linear regression with stochastic gradient descent : R2 score : " + str(R2_Sgd_Sale_Pred))
print("Decision Tree without GridsearchCV : R2 score : " + str(R2_DT_WoutGCV_Sale_Pred))
print("Decision Tree with GridsearchCV : R2 score : " + str(R2_DT_WithGCV_Sale_Pred))
print("Random Forest with GridsearchCV : R2 score : "+ str(R2_RF_WithGCV_Sale_Pred))
Transaction Prediction: Different Models: Metrics : RMSE </u>
In [171]:
print("Linear regression : RMSE : " + str(RMSE_Lin_Reg_Tran_Pred))
print("Linear regression with Stochastic Gradient Descent : RMSE : " + str(RMSE_Sgd_Tran_Pred))
print("Decision Tree without GridsearchCV : RMSE : " + str(RMSE_DT_WoutGCV_Tran_Pred))
print("Decision Tree with GridsearchCV : RMSE : " + str(RMSE_DT_WithGCV_Tran_Pred))
print("Random Forest with GridsearchCV : RMSE : "+ str(RMSE_RF_WithGCV_Tran_Pred))
Sales Prediction: Different Models: Metrics: RMSE </u>
In [172]:
print("Linear regression : RMSE : " + str(RMSE_Lin_Reg_Tran_Pred))
print("Linear regression with stochastic gradient descent: RMSE : " + str(RMSE_Sgd_Tran_Pred))
print("Decision Tree without GridsearchCV : RMSE : " + str(RMSE_DT_WoutGCV_Tran_Pred))
print("Decision Tree with GridsearchCV : RMSE : " + str(RMSE_DT_WithGCV_Tran_Pred))
print("Random Forest with GridsearchCV : RMSE : "+ str(RMSE_RF_WithGCV_Tran_Pred))
Conclusion¶
In [173]:
print("The best regression model for predicting transaction volume & Sales is Random Forest with GridsearchCV")
print("The R2 score of this model for predicting transaction volume is " + str(R2_RF_WithGCV_Tran_Pred))
print("The R2 score of this model for predicting Sales is " + str(R2_RF_WithGCV_Sale_Pred))
print("The RMSE value of this model for predicting transaction volume is " + str(RMSE_RF_WithGCV_Tran_Pred))
print("The RMSE value of this model for predicting sales is " + str(RMSE_RF_WithGCV_Tran_Pred))